﻿using System;
using System.Collections.Generic;
using System.Configuration;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Elderos.TextSearchers;
using Elderos.Utils.Logging;

namespace SynonymFrequencyCalculator
{
    class Program
    {
        private static string _ontologyString = ConfigurationManager.ConnectionStrings["Freebase"].ConnectionString;

        public static void Main(string[] args)
        {
            var stringLoader = new OntologyStringLoader(_ontologyString);
            var searcher = new RabinKarpSearcher(12);
            searcher.Initialize(stringLoader, null);

            int totalTexts;
            var results = CountSynonyms(searcher, out totalTexts);

            Logger.Info("Saving results.");
            Logger.Info("Total texts: " + totalTexts); 
            using (var output = File.CreateText("synonymFrequencies.txt"))
            foreach (var result in results)
            {
                output.WriteLine(result.Key + "\t" + (result.Value*1.0/totalTexts).ToString("0.#######", CultureInfo.InvariantCulture));
            }
            Logger.Info("Results saved.");
        }

        private static Dictionary<string, int> CountSynonyms(RabinKarpSearcher searcher, out int totalTexts)
        {
            var results = new Dictionary<string, int>(StringComparer.InvariantCultureIgnoreCase);

            var textManager = new DbTextManager();


            totalTexts = 0;

            string text = textManager.GetNextText();
            while (text != null)
            {
                List<Position> positions = searcher.Search(text)
                    .Where(x => x.Start == 0 || char.IsWhiteSpace(text[x.Start - 1]))
                    .Where(x => x.Start + x.Length >= text.Length || char.IsWhiteSpace(text[x.Start + x.Length]))
                    .Where(x =>
                               {
                                   string synonym = text.Substring(x.Start, x.Length);
                                   return synonym == synonym.ToLower();
                               })
                    .ToList();

                var synonyms = new HashSet<string>(StringComparer.InvariantCultureIgnoreCase);

                foreach (var position in positions)
                {
                    string syno = text.Substring(position.Start, position.Length);
                    synonyms.Add(syno);
                }

                foreach (var syno in synonyms)
                {
                    if (results.ContainsKey(syno))
                        results[syno]++;
                    else results.Add(syno, 1);
                }

                totalTexts++;
                if (totalTexts%1000 == 0)
                    Logger.Info("Total " + totalTexts + " texts done.");

                text = textManager.GetNextText();
            }
            return results;
        }
    }
}
